# -*- coding: utf-8 -*-
import datetime
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from ArticleSpider.items import LagouJobItemLoader, LagouJobItem
from ArticleSpider.utils.common import get_md5


class LagouSpider(CrawlSpider):
    name = 'lagou'
    allowed_domains = ['lagou.com']
    start_urls = ['https://www.lagou.com/']

    custom_settings = {
        "COOKIES_ENABLED": False,
        "DOWNLOAD_DELAY": 1,
        'DEFAULT_REQUEST_HEADERS': {
            'Accept': 'application/json, text/javascript, */*; q=0.01',
            'Accept-Encoding': 'gzip, deflate, br',
            'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
            'Connection': 'keep-alive',
            'Cookie': 'user_trace_token=20200301162938-a3399158-edd3-43d5-90af-fefd1e3e8667; _ga=GA1.2.1783033281.1583051379; LGUID=20200301162939-09b59aab-e953-4d36-a16e-62a496db72d3; Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1583051457,1583068168,1583158271,1583232963; _gid=GA1.2.718271679.1583051458; index_location_city=%E5%85%A8%E5%9B%BD; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%22170953857cda1-048788a6d4b4aa-4c302879-1049088-170953857ce1%22%2C%22%24device_id%22%3A%22170953857cda1-048788a6d4b4aa-4c302879-1049088-170953857ce1%22%7D; lagou_utm_source=B; JSESSIONID=ABAAAECAAFDAAEHA0B29907EB43E9799F996ED3B9C29A74; WEBTJ-ID=20200303185602-170a009d0b50-0b0c35c653935e-4c302879-1049088-170a009d0b73ce; LGSID=20200303185603-d4a686f5-6db3-496c-9161-ca2619f98ad7; PRE_UTM=; PRE_HOST=www.baidu.com; PRE_SITE=https%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3Do-PaUcN9Fok9C6J-lJHIiAg48vjA0f7wpaj6Ei1ohh7%26wd%3D%26eqid%3D9e87f61a000afa0f000000025e5e37b8; PRE_LAND=https%3A%2F%2Fwww.lagou.com%2F; LGRID=20200303185622-40f33d6f-6bcf-4b0e-a9d3-d40e7bcd9afc; X_HTTP_TOKEN=895abfdbb11993ba07923238510bbd9b3d6b9f9f53; Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1583232971; _gat=1; TG-TRACK-CODE=index_navigation; SEARCH_ID=fa79ebbe459f4731962360c9238bdf97',
            # 'Cookie':'GeeTestUser=5f098be5f2291618258d2e66b4040839',
            'Host': 'www.lagou.com',
            'Origin': 'https://www.lagou.com',
            'Referer': 'https://www.lagou.com/',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36'
        }
    }

    custom_settings = {
        "DOWNLOAD_DELAY": 3
    }

    rules = (
        # Rule(LinkExtractor(allow=("zhaopin/.*",)), follow=True),True
        # Rule(LinkExtractor(allow=("gongsi/j\d+.html",)), follow=True),
        Rule(LinkExtractor(allow=r'zhaopin/.*'), follow=True),
        Rule(LinkExtractor(allow=r'gongsi/\d+.html'), follow=True),
        Rule(LinkExtractor(allow=r'jobs/\d+.html'), callback='parse_item', follow=True),
    )

    # def parse_start_url(self, response):
    #     return []
    #
    #
    # def process_results(self, response, results):
    #     return results

    @staticmethod
    def parse_item(response):
        # 解析拉钩网职位
        item_loader = LagouJobItemLoader(item=LagouJobItem(), response=response)
        item_loader.add_css("title", ".job-name::attr(title)")
        item_loader.add_value("url", response.url)
        item_loader.add_value("usr_object_id", get_md5(response.url))
        item_loader.add_css("salary", ".job_request .salary::text")
        item_loader.add_xpath("job_city", "//*[@class='job_request']/h3/span[2]/text()")
        item_loader.add_xpath("work_years", "//*[@class='job_request']/h3/span[3]/text()")
        item_loader.add_xpath("degree_need", "//*[@class='job_request']/h3/span[4]/text()")
        item_loader.add_xpath("job_type", "//*[@class='job_request']/h3/span[5]/text()")

        item_loader.add_css("tags", ".position-label li::text")
        item_loader.add_css("publish_time", ".publish_time::text")
        item_loader.add_css("job_advantage", ".job-advantage p::text")
        item_loader.add_css("job_desc", ".job_bt div")
        item_loader.add_css("job_addr", ".work_addr")
        item_loader.add_css("company_name", "#job_company dt a img::attr(alt)")
        item_loader.add_css("company_url", "#job_company dt a::attr(href)")
        item_loader.add_value("crawl_time", datetime.datetime.now())

        job_item = item_loader.load_item()

        return job_item
